import json
import time
import pymysql
import requests
from lxml import etree


class Market():
    def __init__(self):
        self.start_url = 'http://cs.ganji.com/zufang/b3/pn{}/'
        self.header = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.90 Safari/537.36'
        }

    def get_url_list(self):
        url_list = []
        for i in range(1, 11):
           url_list.append(self.start_url.format(i))

        return url_list

    def parse_url(self, url):
        response = requests.get(url, headers=self.header)
        r = response.content.decode()
        html = etree.HTML(r)
        # print(html)
        return html

    def get_content_list(self, html):
        content_list = []
        div_list = html.xpath('//div[@class="f-list-item ershoufang-list"]')

        for i in div_list:
            item = {}
            item["URL"] = i.xpath('.//dd[@class="dd-item title"]/a/@href')
            # print(type(item["URL"]))
            address1 = i.xpath('.//dd[@class="dd-item address"]//a[1]/text()')
            # print(address1)
            # print(type(address1))
            # address2 = i.xpath('.//dd[@class="dd-item address"]//a[2]/span/text()')
            # address = address1 + address2
            item["address"] = address1
            price1 = i.xpath('.//div[@class="price"]/span[1]/text()')[0]
            # price2 = i.xpath('.//div[@class="price"]/span[2]/text()')[0]
            # price = price1 + price2
            item["prices"] = price1
            # print(item)
            content_list.append(item)
        # print(content_list)
        return content_list

    def save_content_list(self, content_list):
        db = pymysql.connect("127.0.0.1", "root", "root", "market")
        cursor = db.cursor()

        for it in content_list:
            url = it["URL"]
            address = it["address"]
            # print(address)
            # print(type(address))
            prices = it["prices"]
            # print(prices)
            # print('url',type(url))
            # print(url)
            # print('address',type(address))
            # print(address)
            # print('prices',type(prices))
            sql = ("INSERT INTO aaa(url,addr,price)" "VALUES(%s,%s,%s)")
            # sql = ("INSERT INTO aaa(url,addr,price)" "VALUES(%s,%s,%s)")
            data = (url, address, prices)
            # data = (url, address, prices)
            cursor.execute(sql, data)
            db.commit()
        db.close()
        time.sleep(5)




    def run(self):
        url_list = self.get_url_list()

        for url in url_list:
            html = self.parse_url(url)

            content_list = self.get_content_list(html)

            self.save_content_list(content_list)









if __name__ == '__main__':
    mark = Market()
    mark.run()